



import ast
import re

import requests

import pandas as pd
import base64
import json
import os
import shutil
import time
import random
from collections import OrderedDict
from urllib.parse import quote_plus
import redis
import requests
from sqlalchemy import create_engine
from DrissionPage import ChromiumPage
from DrissionPage import ChromiumOptions
from DrissionPage.common import Settings
from DrissionPage.common import Keys
from DrissionPage.common import Actions
import pymysql
from DrissionPage.errors import *

import datetime

from mysql_ini import mysql_config_ini, redis_config_ini

"""
https://customer.xiaohongshu.com/api/cas/customer/web/qr-code?service=https:%2F%2Fcreator.xiaohongshu.com&qr_code_id=68c517395516719772176435"""

class DY_XPIDER:
    def __init__(self,goods_url):
        self.CO = ChromiumOptions()  ##创建chrome实例
        # self.user_id = self.user_input()  ##用户
        self.user_id = str('local_douyin_01_test')  ##用户
        self.goods_url = goods_url
        self.PORT = self.port_input()  ##chrome端口
        self.CO.set_paths(browser_path="browser_path='/Applications/Google Chrome.app/Contents/MacOS/Google Chrome")  ##chrome启动路径
        self.CO.set_local_port(self.PORT)  ##生成的端口 这里是手动生成
        self.CO.set_user_data_path(os.getcwd() + '/user_info_dir/{}'.format(self.user_id))  ##用户chrome文件夹
        # self.CO.arguments.append('--headless')
        self.ch = ChromiumPage(self.CO)


        self.ch.set.window.max()  ##chrome窗口最大化
        # self.ac = Actions(self.ch)
        # self.fans = self.xhs_user_info_requests()

        self.redis_conn = redis.StrictRedis(host=redis_config_ini()["host"], port=redis_config_ini()["port"],
                                            password=redis_config_ini()["password"], db=redis_config_ini()["database"])

        # self.redis_conn = redis.StrictRedis(host='r-bp1u7oiaxwfpof7014.redis.rds.aliyuncs.com', port=6379,
        #                                     password='GHGOAPSqVaMWZJf6JCT', db=0)


    """根据用户ID查出端口号"""

    def port_fe(self):
        sql = "select spider_user_id,spider_port,spider_login_code from dy_creator_center where spider_user_id='{}'".format(
            self.user_id)
        spider_config = self.pymysql_fetchall(sql)
        port = []
        for i in spider_config:
            port.append(i[1])

        return ''.join(port)

    """自动输入端口号"""

    def port_input(self):

        port1 = random.randint(5000, 9000)

        cc = self.port_jiance()
        while True:
            cc = self.port_jiance()
            if cc == "用户存在":
                # print("当前用户存在")
                port_ = self.port_fe()
                # print(port_)
                return port_
            if cc == "端口不存在":
                # print(port1)
                return port1
            else:
                return self.PORT

    """自动随机生成端口 并进行检测"""

    def port_jiance(self):
        port1 = random.randint(5000, 9000)

        sql = "select spider_user_id,spider_port,spider_login_code from dy_creator_center"
        spider_config = self.pymysql_fetchall(sql)
        port_list = []
        # print(spider_config)
        for i in spider_config:
            port_list.append(i[1])
        user_exists = self.user_state()
        # print(user_exists, '************')
        if user_exists == "用户不存在":
            if port1 not in port_list:
                return "端口不存在"
            else:
                return "端口存在"
        else:
            return '用户存在'

    """查询数据库"""

    def pymysql_fetchall(self, sql):

        con = pymysql.connect(host=mysql_config_ini()["host"], port=mysql_config_ini()["port"], user=mysql_config_ini()["user"],
                              passwd=mysql_config_ini()["password"], db=mysql_config_ini()["database"], charset='utf8')
        # 获取操作数据的对象 cursor
        cursor = con.cursor()
        cursor.execute(sql)
        sql_result = cursor.fetchall()
        # 提交事务
        con.commit()
        # 关闭Cursor
        cursor.close()
        # 关闭链接
        con.close()

        return sql_result

    # def pymysql_fetchall(self, sql):
    #
    #     con = pymysql.connect(host='rm-bp17709mm0o30s13r.mysql.rds.aliyuncs.com', port=3306, user='warehouse',
    #                           passwd='QlciCnfTB9NtLSC', db='data-warehouse', charset='utf8')
    #     # 获取操作数据的对象 cursor
    #     cursor = con.cursor()
    #     cursor.execute(sql)
    #     sql_result = cursor.fetchall()
    #     # 提交事务
    #     con.commit()
    #     # 关闭Cursor
    #     cursor.close()
    #     # 关闭链接
    #     con.close()
    #
    #     return sql_result

    """从数据库判断用户是否存在"""

    def user_state(self):
        sql = "select spider_user_id,spider_port,spider_login_code from dy_creator_center"
        result = self.pymysql_fetchall(sql)
        user_list = []
        for i in result:
            user_list.append(i[0])
        # print(user_list)
        if self.user_id in user_list:
            # print("存在")
            return "用户存在"
        else:
            return "用户不存在"

    """判断用户是否在数据库中 不在为新用户 在的话为老用户"""

    def user_info(self):
        engine2 = self.mysql_config()
        data_frame = pd.DataFrame([{"spider_user_id": str(self.user_id), "spider_port": str(self.PORT)}])
        # print(data_frame)
        # pd.io.sql.to_sql(data_frame, 'xhs_user_info', engine2, schema='live_data',if_exists='append',index=False)
        return data_frame


    # def mysql_config(self):
    #     user = 'live_data'
    #     password = quote_plus('CJMg2tsHg7sj6bo0UTKq0a4T!fPDpJ')
    #     # passwd ='merchantsasd123!@%&'
    #     host = 'rm-bp17r99475xuauk63yo.mysql.rds.aliyuncs.com'
    #     # port1 ='3306'
    #     dbname2 = 'live_data'
    #     engine2 = create_engine(f"mysql+pymysql://{user}:{password}@{host}:3306/{dbname2}?charset=utf8mb4")
    #     return engine2


    def mysql_config(self):
        user = mysql_config_ini()["user"]
        password = quote_plus(mysql_config_ini()["password"])
        # passwd ='merchantsasd123!@%&'
        host = mysql_config_ini()["host"]
        # port1 ='3306'
        dbname2 = mysql_config_ini()["database"]
        engine2 = create_engine(f"mysql+pymysql://{user}:{password}@{host}:3306/{dbname2}?charset=utf8mb4")
        return engine2

    # def mysql_config(self):
    #     user = 'warehouse'
    #     # password = quote_plus('QlciCnfTB9NtLSC')
    #     password = 'QlciCnfTB9NtLSC'
    #     # passwd ='merchantsasd123!@%&'
    #     host = 'rm-bp17709mm0o30s13r.mysql.rds.aliyuncs.com'
    #     # port1 ='3306'
    #     dbname2 = 'data-warehouse'
    #     engine2 = create_engine(f"mysql+pymysql://{user}:{password}@{host}:3306/{dbname2}?charset=utf8mb4")
    #     return engine2

    """修改用户登录状态的mysql"""

    def pymysql_update_dy_user_info(self, sql,):

        con = pymysql.connect(host=mysql_config_ini()["host"], port=mysql_config_ini()["port"], user=mysql_config_ini()["user"],
                              passwd=mysql_config_ini()["password"], db=mysql_config_ini()["database"], charset='utf8')
        # 获取操作数据的对象 cursor
        cursor = con.cursor()
        cursor.execute(sql)
        # sql_result = cursor.fetchall()
        # 提交事务
        con.commit()
        # 关闭Cursor
        cursor.close()
        # 关闭链接
        con.close()

        return "修改成功"



    def delete_folder_random(self,random_id):
        try:
            shutil.rmtree(os.getcwd() + "\\user_info_dir\\" + random_id)
            print(f"成功删除文件夹")
        except OSError as e:
            print(f"删除文件夹失败")


    def pymysql_update(self, sql,num,user_id):

        con = pymysql.connect(host=mysql_config_ini()["host"], port=mysql_config_ini()["port"], user=mysql_config_ini()["user"],
                              passwd=mysql_config_ini()["password"], db=mysql_config_ini()["database"], charset='utf8')
        # 获取操作数据的对象 cursor
        cursor = con.cursor()
        cursor.execute(sql,(num,user_id))
        sql_result = cursor.fetchall()
        # 提交事务
        con.commit()
        # 关闭Cursor
        cursor.close()
        # 关闭链接
        con.close()

        return sql_result


    """获取cookies"""
    def get_cookies(self):
        cookies_str = ''
        for i in self.ch.cookies(all_domains=False, all_info=True):
            my_ordered_dict = OrderedDict(i)

            # 取出前两个键值对
            first_two_items = list(my_ordered_dict.items())[:2]
            key_value_str = ''
            count_key_value = 0
            for i in first_two_items:
                count_key_value = count_key_value + 1
                if count_key_value == 1:
                    key_value_str = key_value_str + i[1] + '='
                else:
                    key_value_str = key_value_str + i[1]
            cookies_str = cookies_str + key_value_str + ";"
        return cookies_str


        # 打印简化后的数据
        # print(simplified_data)
        return simplified_data

    def live_data_config(self):
        user = 'live_data'
        password = quote_plus('CJMg2tsHg7sj6bo0UTKq0a4T!fPDpJ')
        # passwd ='merchantsasd123!@%&'
        host = 'rm-bp17r99475xuauk63yo.mysql.rds.aliyuncs.com'
        # port1 ='3306'
        dbname2 = 'live_data'
        engine2 = create_engine(f"mysql+pymysql://{user}:{password}@{host}:3306/{dbname2}?charset=utf8mb4")
        return engine2

    def dy_creator_center_spider(self):
        for gurl in self.goods_url:
            gurl_id = str(gurl).split("https://haohuo.jinritemai.com/ecommerce/trade/detail/index.html?id=")[1].split('&origin_type=pc_buyin_selection_decision')[0]
            print(gurl_id)
            print('用户:',self.user_id, '此次的端口是:',self.PORT)
            # time.sleep(4)
            self.ch.get('https://buyin.jinritemai.com/dashboard/merch-picking-library?pre_universal_page_params_id=&universal_page_params_id=9e4f8ae3-312b-4d4e-864d-7315a8eeec21')
            # self.ch.get('https://www.baidu.com')
            time.sleep(1)
            user_it_exist = self.user_state()
            data_frame = self.user_info()
            if user_it_exist == "用户不存在":
                data_frame["spider_login_code"] = '0'
                pd.io.sql.to_sql(data_frame, 'dy_creator_center', self.mysql_config(), schema='data-warehouse',
                                 if_exists='append', index=False)

            # print("走到这里了")
            login_url_is = self.ch.url

            """点击搜索输入框"""

            print(gurl)
            goods_url = gurl
            self.ch.ele('xpath://*[@id="rc_select_0"]').input('{}\n'.format(goods_url))
            self.ch.ele('xpath://*[@id="SearchBarWrapper"]/div/div[1]/button/span').click()

            self.ch.listen.start('material_list')

            product_id = []
            promotion_id = []
            for packet in self.ch.listen.steps(timeout=2):
                goods_detail = packet.response.body
                product_id.append(goods_detail["data"]["promotions"][0]["product_id"])
                promotion_id.append(goods_detail["data"]["promotions"][0]["promotion_id"])
            # print(product_id)
            # print(promotion_id)
            product_id = product_id[0]
            promotion_id = promotion_id[0]
            url = "https://buyin.jinritemai.com/dashboard/merch-picking-library/merch-promoting?commodity_id={}&commodity_location=1&id={}&pick_first_source=%E7%99%BE%E5%BA%94&pick_second_source=%E9%80%89%E5%93%81%E5%B9%BF%E5%9C%BA&pick_third_source=search&pick_source_id=&product_id={}".format(promotion_id,promotion_id,product_id)
            self.ch.get(url)

            """商品标题"""
            goods_title = self.ch.ele('xpath://*[@id="app"]/div/div/div[1]/div[2]/div[1]/div[1]/div[1]/span')
            print(goods_title.text)
            """点击带货内容模块"""
            time.sleep(1)
            self.ch.ele('xpath://*[@id="rc-tabs-0-tab-content"]').click()
            time.sleep(1)

            desc_list = []
            for i in range(1,8):
                time.sleep(0.5)
                desc = self.ch.ele('xpath://*[@id="rc-tabs-0-panel-content"]/div/div[2]/div[2]/div/div/div/div[{}]/div/div[1]/div/div[2]/div'.format(i))
                desc_txt = desc.text
                desc_list.append(desc_txt)
            desc_match = ' '.join(desc_list)
            pattern = r'#\w+'
            matches = re.findall(pattern, desc_match)

            unique_set = set(matches)

            # 如果需要列表格式，将集合转换回列表
            unique_list_desc = list(unique_set)
            if len(unique_list_desc) >5:
                unique_list_desc = unique_list_desc[:5]
            else:
                unique_list_desc = unique_list_desc

            goods_result = [{"goods_unique_id":gurl_id,"title":goods_title.text,"desc":unique_list_desc,"goods_url":gurl}]
            goods_frame = pd.DataFrame(goods_result)
            print(goods_frame)
            goods_frame = goods_frame.astype("str")
            pd.io.sql.to_sql(goods_frame, 'douyin_goods_detail', self.live_data_config(), schema='live_data',if_exists='append', index=False)
        self.ch.close()

goods_url_list = ['https://haohuo.jinritemai.com/ecommerce/trade/detail/index.html?id=3662185801577556295&origin_type=pc_buyin_selection_decision',
'https://haohuo.jinritemai.com/ecommerce/trade/detail/index.html?id=3716358566278463649&origin_type=pc_buyin_selection_decision',
'https://haohuo.jinritemai.com/ecommerce/trade/detail/index.html?id=3658173717915263291&origin_type=pc_buyin_selection_decision',
'https://haohuo.jinritemai.com/ecommerce/trade/detail/index.html?id=3721695112544256533&origin_type=pc_buyin_selection_decision',
'https://haohuo.jinritemai.com/ecommerce/trade/detail/index.html?id=3663459362460011467&origin_type=pc_buyin_selection_decision',
'https://haohuo.jinritemai.com/ecommerce/trade/detail/index.html?id=3689547499447648440&origin_type=pc_buyin_selection_decision',
'https://haohuo.jinritemai.com/ecommerce/trade/detail/index.html?id=3708265537944551847&origin_type=pc_buyin_selection_decision',
'https://haohuo.jinritemai.com/ecommerce/trade/detail/index.html?id=3660969140405014771&origin_type=pc_buyin_selection_decision',
'https://haohuo.jinritemai.com/ecommerce/trade/detail/index.html?id=3554455453637418026&origin_type=pc_buyin_selection_decision',
'https://haohuo.jinritemai.com/ecommerce/trade/detail/index.html?id=3696309607594066074&origin_type=pc_buyin_selection_decision']

dy_center = DY_XPIDER(goods_url=goods_url_list)
aa = dy_center.dy_creator_center_spider()
















##QlciCnfTB9NtLSC

'https://buyin.jinritemai.com/dashboard/merch-picking-library/merch-promoting?commodity_id=3613115653025850885&commodity_location=1&id=3613115653025850885&pick_first_source=%E7%99%BE%E5%BA%94&pick_second_source=%E9%80%89%E5%93%81%E5%B9%BF%E5%9C%BA&pick_third_source=search&pick_source_id=&product_id=3612638209428818866&shop_id=86365766&search_id=68e02a10c2d9e8b3bc940b7a2374767568123fe89a5c01e329e49513e21a02627564fe1d619977e38d5b2015b715e9826df4cb3167304cab130ed4bc9d5ebead41964bb2fae66b98311a837aab80a57a&log_pb=20241225142112DFCD0C525F59CDF661E8&session_id=2240ef2d8a855155c5a83bb0482ffccf197cd99dd99017347d914442b8d79bf10fb34e65d709158faa3aa3e3600ba132a6ce5a4e0e2eece3be9d899ab90db2f427cc8244d5f0707d8e9ae4481e40d5e5&recommend_reason_type=&recommend_reason_text=&is_outside_log=1&only_query_params=1&universal_append=1&pre_universal_unit_params=%257B%2522product_id%2522%253A%25223612638209428818866%2522%252C%2522commodity_id%2522%253A%25223613115653025850885%2522%252C%2522commodity_location%2522%253A1%252C%2522search_id%2522%253A%252268e02a10c2d9e8b3bc940b7a2374767568123fe89a5c01e329e49513e21a02627564fe1d619977e38d5b2015b715e9826df4cb3167304cab130ed4bc9d5ebead41964bb2fae66b98311a837aab80a57a%2522%252C%2522log_pb%2522%253A%252220241225142112DFCD0C525F59CDF661E8%2522%257D&decision_enter_from=pc.selection_square.recommend_main&btm_ppre=a0.b0.c0.d0&btm_pre=a10091.b24215.c962900.d595042_i0&btm_show_id=f86092b4-f548-427e-925f-75b209febe6c&pre_universal_page_params_id=9e4f8ae3-312b-4d4e-864d-7315a8eeec21&universal_page_params_id=f38ba9cc-27b4-49ad-841b-a465d35308e5'
'https://buyin.jinritemai.com/dashboard/merch-picking-library/merch-promoting?commodity_id=3613115653025850885&commodity_location=1&id=3613115653025850885&pick_first_source=%E7%99%BE%E5%BA%94&pick_second_source=%E9%80%89%E5%93%81%E5%B9%BF%E5%9C%BA&pick_third_source=search&pick_source_id=&product_id=3612638209428818866'
"""带货内容那"""
####https://buyin.jinritemai.com/pc/selection/decision/pack_detail?verifyFp=verify_m522tdga_L4EiLq6J_ZLPd_44UD_AzgJ_hXs9VhgzPrdb&fp=verify_m522tdga_L4EiLq6J_ZLPd_44UD_AzgJ_hXs9VhgzPrdb&msToken=5ghV06vHuCFjSdyenLye8uEh1oJOj_LJr1_LKd3gU2EXilLdpH8LDygVQYbKzRiuMpeUtJoxnxLZGvK8V3mJp4QKdw8xLIeAiAfvwj5GJOHBCW-95YiM8voOIRkMz_-VVeGKhQBer8nAPY8N6tIHo9nkVKnDiXQWa7BRp3PKVXo%3D&a_bogus=xJ0Rgqy7YpQVFpCS8CTwefAUEC6ANTSyfMiKb7eT7FPuY70Ou9BQkTEgaNLl%2F6kmNWB1ku-7Px49ExnbB9kt2FnkzmkfuN7yh0ACnUXo%2FHkhT4vZJqmiSJbxFiPT0W4Y%2F%2FdUNBiX60zwI2O3hHniW5la75za5QYDbrqSd%2Fmbc9WYDC6PY3rbOZEAEj7F

# aa = ['#麂皮刷', '#鞋刷', '#雪地靴', '#勃肯鞋', '#擦鞋神器', '#马丁靴', '#刷鞋神器', '#麂皮绒']
# print(aa[:5])

#####搜索内容
"""https://buyin.jinritemai.com/pc/selection/common/material_list?msToken=IOrVxj8CTPjsT-A7a6FVs00EUqZlElrO9YD2ZVatYnPaLLCSPY5dnBqfGl7V1tBFDMB_lk0roX09RLuJ9uB_h2yVrApiADGUmorZoNJTr3-DJoXv2dADYYzBHsmxtDS9sWmIrxS5mk2sQnnmPRVeSjv17RG6CEv9jiHq9QuJnTgyVy5kotjC&a_bogus=DyURhtSLEZ55FplSYCmF9l3l6ID%2FNP8ybPTQbYMTSlbCYXFPydBxiSDaJPicIHH7NWp3koZ7rkS7udVbPxpl0qOpLmhkuKtSwU55nU8Lgqq3b0i0gHyFCzDFLwMe0mvwa591i1RRXsBrIjxA9HVYWdMa75PHQcDdSHZyDZi97EWhfA8kwo-TOCkpiLJOUvO7&verifyFp=verify_m522tdga_L4EiLq6J_ZLPd_44UD_AzgJ_hXs9VhgzPrdb&fp=verify_m522tdga_L4EiLq6J_ZLPd_44UD_AzgJ_hXs9VhgzPrdb"""





"""https://buyin.jinritemai.com/dashboard/merch-picking-library?pre_universal_page_params_id=&universal_page_params_id=9e4f8ae3-312b-4d4e-864d-7315a8eeec21"""


# url = 'https://haohuo.jinritemai.com/ecommerce/trade/detail/index.html?id=3662185801577556295&origin_type=pc_buyin_selection_decision'
#
# print(url.split("id="))